{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f34b3bc5",
   "metadata": {},
   "source": [
    "# Statically Quantize Roberta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b353d22",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd8a69e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import glue_compute_metrics\n",
    "import sklearn\n",
    "from sklearn.metrics import f1_score\n",
    "from tqdm import tqdm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e4e773cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dynamic_quant_roberta import QuantRobertaForSequenceClassification\n",
    "from transformers import RobertaForSequenceClassification, AutoTokenizer\n",
    "from transformers.data.metrics import simple_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8ce4361a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at textattack/roberta-base-MRPC were not used when initializing QuantRobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing QuantRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing QuantRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at textattack/roberta-base-MRPC were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "qmodel = QuantRobertaForSequenceClassification.from_pretrained('textattack/roberta-base-MRPC')\n",
    "model = RobertaForSequenceClassification.from_pretrained('textattack/roberta-base-MRPC')\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('textattack/roberta-base-MRPC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ff68ee2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset glue (/Users/oliver/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset('glue', 'mrpc', split='validation')\n",
    "# dataset = load_dataset('glue', 'mrpc', split='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "03bf91c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s>He said the foodservice pie business doesn \\'t fit the company\\'s long-term growth strategy.</s></s>\" The foodservice pie business does not fit our long-term growth strategy.</s>'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenizer(dataset[0]['sentence1'], dataset[0]['sentence2'])['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9d14ed15",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /Users/oliver/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-7ed1324c21291143.arrow\n"
     ]
    }
   ],
   "source": [
    "def encode(examples):\n",
    "    return tokenizer(examples['sentence1'], examples['sentence2'], truncation=True, padding='max_length')\n",
    "\n",
    "dataset = dataset.map(encode, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "971cb824",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /Users/oliver/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fc668b1e09db634.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'attention_mask', 'labels'],\n",
       "    num_rows: 408\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.map(lambda examples: {'labels': examples['label']}, batched=True)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "672a12bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "581864b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = torch.utils.data.DataLoader(dataset, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "858b3f5e",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5feb8b27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_model(model, dataloader):\n",
    "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "    model.eval()\n",
    "    preds = None\n",
    "\n",
    "    for i, batch in enumerate(tqdm(dataloader)):\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**batch)\n",
    "            tmp_eval_loss, logits = outputs[:2]\n",
    "            loss = outputs[0]\n",
    "        if preds is None:\n",
    "            preds = logits.detach().cpu().numpy()\n",
    "            out_label_ids = batch['labels'].detach().cpu().numpy()\n",
    "        else:\n",
    "            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n",
    "            out_label_ids = np.append(out_label_ids, batch['labels'].detach().cpu().numpy(), axis=0)\n",
    "        if i % 10 == 0:\n",
    "    #         print(f\"loss: {loss}\")\n",
    "            pass\n",
    "\n",
    "    preds = np.argmax(preds, axis=1)\n",
    "\n",
    "    print(f'accuracy: {simple_accuracy(preds, out_label_ids)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "919b4ae9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                                                                                                              | 0/13 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 13/13 [04:05<00:00, 18.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9117647058823529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "/Users/oliver/miniforge3/envs/trans-fat/lib/python3.9/site-packages/transformers/data/metrics/__init__.py:36: FutureWarning: This metric will be removed from the library soon, metrics should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/pytorch/text-classification/run_glue.py\n",
      "  warnings.warn(DEPRECATION_WARNING, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "eval_model(model, dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "026c7095",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 13/13 [09:53<00:00, 45.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8970588235294118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "eval_model(qmodel, dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "2a2c0ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.backends.quantized.engine = 'qnnpack'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "187d43d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dynamic_model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "a7a05da0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 54/54 [17:12<00:00, 19.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8684057971014493\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "eval_model(dynamic_model, dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71eecc7c",
   "metadata": {},
   "source": [
    "## PyTorch dynamic quantization under the hood\n",
    "How does pytorch determine the min and max of the range?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "29a181b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0727, -0.0056, -0.0895,  ...,  0.1007,  0.0895, -0.1007],\n",
       "        [-0.0503,  0.2070,  0.0727,  ...,  0.0671,  0.0615,  0.1286],\n",
       "        [ 0.0895,  0.0727, -0.0503,  ..., -0.0447, -0.0056,  0.1119],\n",
       "        ...,\n",
       "        [-0.1846,  0.0168, -0.0336,  ..., -0.0503,  0.1007, -0.1175],\n",
       "        [-0.2517,  0.0447,  0.0615,  ...,  0.0727, -0.1063,  0.0112],\n",
       "        [-0.0503, -0.0839,  0.1007,  ..., -0.1902,  0.0056, -0.0559]],\n",
       "       size=(768, 768), dtype=torch.qint8,\n",
       "       quantization_scheme=torch.per_tensor_affine, scale=0.0055933459661901,\n",
       "       zero_point=0)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dynamic_model.roberta.encoder.layer[0].attention.self.query.weight()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "985cba81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0733, -0.0037, -0.0904,  ...,  0.1033,  0.0894, -0.1026],\n",
       "        [-0.0520,  0.2053,  0.0730,  ...,  0.0648,  0.0631,  0.1287],\n",
       "        [ 0.0869,  0.0704, -0.0509,  ..., -0.0434, -0.0070,  0.1100],\n",
       "        ...,\n",
       "        [-0.1867,  0.0172, -0.0314,  ..., -0.0504,  0.1023, -0.1159],\n",
       "        [-0.2524,  0.0435,  0.0640,  ...,  0.0703, -0.1036,  0.0117],\n",
       "        [-0.0512, -0.0864,  0.1022,  ..., -0.1887,  0.0045, -0.0540]])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[0].attention.self.query.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e084579f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7132)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[0].attention.self.query.weight.data.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "9f372626",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.5972)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[0].attention.self.query.weight.data.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "5a447311",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmin = -0.5972\n",
    "tmax = 0.7132"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "c6dd516a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-128.5, 127.5)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qmax = 127\n",
    "qmin = -128\n",
    "symmetric_qmin = -((qmax - qmin) / 2 + 1)\n",
    "symmetric_qmax = (qmax - qmin) / 2\n",
    "symmetric_qmin, symmetric_qmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "f2c12b58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.005593725490196078"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_scale = max(abs(tmin / symmetric_qmin), abs(tmax / symmetric_qmax))\n",
    "max_scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "0ce4a5a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmin = max_scale*symmetric_qmin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "84d2a76a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmax = max_scale*symmetric_qmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "eee84afa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.718793725490196, 0.7132)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmin, tmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "868b7623",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.005615661668589004"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tmax - tmin) / (qmax - qmin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "85ff0657",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0727, -0.0056, -0.0895,  ...,  0.1007,  0.0895, -0.1007],\n",
       "        [-0.0503,  0.2070,  0.0727,  ...,  0.0671,  0.0615,  0.1287],\n",
       "        [ 0.0895,  0.0727, -0.0503,  ..., -0.0447, -0.0056,  0.1119],\n",
       "        ...,\n",
       "        [-0.1846,  0.0168, -0.0336,  ..., -0.0503,  0.1007, -0.1175],\n",
       "        [-0.2517,  0.0447,  0.0615,  ...,  0.0727, -0.1063,  0.0112],\n",
       "        [-0.0503, -0.0839,  0.1007,  ..., -0.1902,  0.0056, -0.0559]])"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aq = torch.round(model.roberta.encoder.layer[0].attention.self.query.weight.data / max_scale)\n",
    "aq * max_scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "e4e67a37",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'LinearPackedParams' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [91]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdynamic_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mroberta\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mself\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_packed_params\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'LinearPackedParams' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "dynamic_model.roberta.encoder.layer[0].attention.self.query._packed_params[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "d319bd05",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dynamic_quant_ops import tensor_quant_scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "286de1c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "aq, scale = tensor_quant_scale(model.roberta.encoder.layer[0].attention.self.query.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "e40e9ac2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0727, -0.0056, -0.0895,  ...,  0.1007,  0.0895, -0.1007],\n",
       "        [-0.0503,  0.2070,  0.0727,  ...,  0.0671,  0.0615,  0.1286],\n",
       "        [ 0.0895,  0.0727, -0.0503,  ..., -0.0447, -0.0056,  0.1119],\n",
       "        ...,\n",
       "        [-0.1846,  0.0168, -0.0336,  ..., -0.0503,  0.1007, -0.1175],\n",
       "        [-0.2517,  0.0447,  0.0615,  ...,  0.0727, -0.1063,  0.0112],\n",
       "        [-0.0503, -0.0839,  0.1007,  ..., -0.1902,  0.0056, -0.0559]])"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aq * scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "6f084bbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.382255209922794"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(model.roberta.encoder.layer[0].attention.self.query.weight.data.detach().numpy(), 99.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "4a5e0fab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.005615748031496063"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(abs(0.7132), abs(-0.5972)) / (2**(8-1)-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fad1dc25",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = np.load('state_dict.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a63505ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.token_type_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.encoder.layer.0.attention.self.query.weight\n",
      "bert.encoder.layer.0.attention.self.query.bias\n",
      "bert.encoder.layer.0.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.key.weight\n",
      "bert.encoder.layer.0.attention.self.key.bias\n",
      "bert.encoder.layer.0.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.value.weight\n",
      "bert.encoder.layer.0.attention.self.value.bias\n",
      "bert.encoder.layer.0.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.output.dense.weight\n",
      "bert.encoder.layer.0.attention.output.dense.bias\n",
      "bert.encoder.layer.0.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.0.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.0.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.0.intermediate.dense.weight\n",
      "bert.encoder.layer.0.intermediate.dense.bias\n",
      "bert.encoder.layer.0.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.0.output.dense.weight\n",
      "bert.encoder.layer.0.output.dense.bias\n",
      "bert.encoder.layer.0.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.0.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.output.LayerNorm.bias\n",
      "bert.encoder.layer.0.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.0.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.query.weight\n",
      "bert.encoder.layer.1.attention.self.query.bias\n",
      "bert.encoder.layer.1.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.key.weight\n",
      "bert.encoder.layer.1.attention.self.key.bias\n",
      "bert.encoder.layer.1.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.value.weight\n",
      "bert.encoder.layer.1.attention.self.value.bias\n",
      "bert.encoder.layer.1.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.output.dense.weight\n",
      "bert.encoder.layer.1.attention.output.dense.bias\n",
      "bert.encoder.layer.1.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.1.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.1.intermediate.dense.weight\n",
      "bert.encoder.layer.1.intermediate.dense.bias\n",
      "bert.encoder.layer.1.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.1.output.dense.weight\n",
      "bert.encoder.layer.1.output.dense.bias\n",
      "bert.encoder.layer.1.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.1.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.1.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.query.weight\n",
      "bert.encoder.layer.2.attention.self.query.bias\n",
      "bert.encoder.layer.2.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.key.weight\n",
      "bert.encoder.layer.2.attention.self.key.bias\n",
      "bert.encoder.layer.2.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.value.weight\n",
      "bert.encoder.layer.2.attention.self.value.bias\n",
      "bert.encoder.layer.2.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.output.dense.weight\n",
      "bert.encoder.layer.2.attention.output.dense.bias\n",
      "bert.encoder.layer.2.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.2.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.2.intermediate.dense.weight\n",
      "bert.encoder.layer.2.intermediate.dense.bias\n",
      "bert.encoder.layer.2.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.2.output.dense.weight\n",
      "bert.encoder.layer.2.output.dense.bias\n",
      "bert.encoder.layer.2.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.2.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.2.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.query.weight\n",
      "bert.encoder.layer.3.attention.self.query.bias\n",
      "bert.encoder.layer.3.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.key.weight\n",
      "bert.encoder.layer.3.attention.self.key.bias\n",
      "bert.encoder.layer.3.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.value.weight\n",
      "bert.encoder.layer.3.attention.self.value.bias\n",
      "bert.encoder.layer.3.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.output.dense.weight\n",
      "bert.encoder.layer.3.attention.output.dense.bias\n",
      "bert.encoder.layer.3.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.3.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.3.intermediate.dense.weight\n",
      "bert.encoder.layer.3.intermediate.dense.bias\n",
      "bert.encoder.layer.3.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.3.output.dense.weight\n",
      "bert.encoder.layer.3.output.dense.bias\n",
      "bert.encoder.layer.3.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.3.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.3.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.query.weight\n",
      "bert.encoder.layer.4.attention.self.query.bias\n",
      "bert.encoder.layer.4.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.key.weight\n",
      "bert.encoder.layer.4.attention.self.key.bias\n",
      "bert.encoder.layer.4.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.value.weight\n",
      "bert.encoder.layer.4.attention.self.value.bias\n",
      "bert.encoder.layer.4.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.output.dense.weight\n",
      "bert.encoder.layer.4.attention.output.dense.bias\n",
      "bert.encoder.layer.4.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.4.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.4.intermediate.dense.weight\n",
      "bert.encoder.layer.4.intermediate.dense.bias\n",
      "bert.encoder.layer.4.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.4.output.dense.weight\n",
      "bert.encoder.layer.4.output.dense.bias\n",
      "bert.encoder.layer.4.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.4.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.4.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.query.weight\n",
      "bert.encoder.layer.5.attention.self.query.bias\n",
      "bert.encoder.layer.5.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.key.weight\n",
      "bert.encoder.layer.5.attention.self.key.bias\n",
      "bert.encoder.layer.5.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.value.weight\n",
      "bert.encoder.layer.5.attention.self.value.bias\n",
      "bert.encoder.layer.5.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.output.dense.weight\n",
      "bert.encoder.layer.5.attention.output.dense.bias\n",
      "bert.encoder.layer.5.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.5.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.5.intermediate.dense.weight\n",
      "bert.encoder.layer.5.intermediate.dense.bias\n",
      "bert.encoder.layer.5.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.5.output.dense.weight\n",
      "bert.encoder.layer.5.output.dense.bias\n",
      "bert.encoder.layer.5.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.5.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.5.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.query.weight\n",
      "bert.encoder.layer.6.attention.self.query.bias\n",
      "bert.encoder.layer.6.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.key.weight\n",
      "bert.encoder.layer.6.attention.self.key.bias\n",
      "bert.encoder.layer.6.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.value.weight\n",
      "bert.encoder.layer.6.attention.self.value.bias\n",
      "bert.encoder.layer.6.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.output.dense.weight\n",
      "bert.encoder.layer.6.attention.output.dense.bias\n",
      "bert.encoder.layer.6.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.6.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.6.intermediate.dense.weight\n",
      "bert.encoder.layer.6.intermediate.dense.bias\n",
      "bert.encoder.layer.6.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.6.output.dense.weight\n",
      "bert.encoder.layer.6.output.dense.bias\n",
      "bert.encoder.layer.6.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.6.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.6.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.query.weight\n",
      "bert.encoder.layer.7.attention.self.query.bias\n",
      "bert.encoder.layer.7.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.key.weight\n",
      "bert.encoder.layer.7.attention.self.key.bias\n",
      "bert.encoder.layer.7.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.value.weight\n",
      "bert.encoder.layer.7.attention.self.value.bias\n",
      "bert.encoder.layer.7.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.output.dense.weight\n",
      "bert.encoder.layer.7.attention.output.dense.bias\n",
      "bert.encoder.layer.7.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.7.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.7.intermediate.dense.weight\n",
      "bert.encoder.layer.7.intermediate.dense.bias\n",
      "bert.encoder.layer.7.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.7.output.dense.weight\n",
      "bert.encoder.layer.7.output.dense.bias\n",
      "bert.encoder.layer.7.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.7.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.7.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.query.weight\n",
      "bert.encoder.layer.8.attention.self.query.bias\n",
      "bert.encoder.layer.8.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.key.weight\n",
      "bert.encoder.layer.8.attention.self.key.bias\n",
      "bert.encoder.layer.8.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.value.weight\n",
      "bert.encoder.layer.8.attention.self.value.bias\n",
      "bert.encoder.layer.8.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.output.dense.weight\n",
      "bert.encoder.layer.8.attention.output.dense.bias\n",
      "bert.encoder.layer.8.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.8.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.8.intermediate.dense.weight\n",
      "bert.encoder.layer.8.intermediate.dense.bias\n",
      "bert.encoder.layer.8.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.8.output.dense.weight\n",
      "bert.encoder.layer.8.output.dense.bias\n",
      "bert.encoder.layer.8.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.8.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.8.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.query.weight\n",
      "bert.encoder.layer.9.attention.self.query.bias\n",
      "bert.encoder.layer.9.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.key.weight\n",
      "bert.encoder.layer.9.attention.self.key.bias\n",
      "bert.encoder.layer.9.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.value.weight\n",
      "bert.encoder.layer.9.attention.self.value.bias\n",
      "bert.encoder.layer.9.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.output.dense.weight\n",
      "bert.encoder.layer.9.attention.output.dense.bias\n",
      "bert.encoder.layer.9.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.9.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.9.intermediate.dense.weight\n",
      "bert.encoder.layer.9.intermediate.dense.bias\n",
      "bert.encoder.layer.9.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.9.output.dense.weight\n",
      "bert.encoder.layer.9.output.dense.bias\n",
      "bert.encoder.layer.9.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.9.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.9.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.query.weight\n",
      "bert.encoder.layer.10.attention.self.query.bias\n",
      "bert.encoder.layer.10.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.key.weight\n",
      "bert.encoder.layer.10.attention.self.key.bias\n",
      "bert.encoder.layer.10.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.value.weight\n",
      "bert.encoder.layer.10.attention.self.value.bias\n",
      "bert.encoder.layer.10.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.output.dense.weight\n",
      "bert.encoder.layer.10.attention.output.dense.bias\n",
      "bert.encoder.layer.10.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.10.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.10.intermediate.dense.weight\n",
      "bert.encoder.layer.10.intermediate.dense.bias\n",
      "bert.encoder.layer.10.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.10.output.dense.weight\n",
      "bert.encoder.layer.10.output.dense.bias\n",
      "bert.encoder.layer.10.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.10.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.10.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.query.weight\n",
      "bert.encoder.layer.11.attention.self.query.bias\n",
      "bert.encoder.layer.11.attention.self.query._input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.key.weight\n",
      "bert.encoder.layer.11.attention.self.key.bias\n",
      "bert.encoder.layer.11.attention.self.key._input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.value.weight\n",
      "bert.encoder.layer.11.attention.self.value.bias\n",
      "bert.encoder.layer.11.attention.self.value._input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.qv_a_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.qv_b_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.av_a_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.self.av_b_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.output.dense.weight\n",
      "bert.encoder.layer.11.attention.output.dense.bias\n",
      "bert.encoder.layer.11.attention.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.attention.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.11.attention.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.layer.11.intermediate.dense.weight\n",
      "bert.encoder.layer.11.intermediate.dense.bias\n",
      "bert.encoder.layer.11.intermediate.dense._input_quantizer._amax\n",
      "bert.encoder.layer.11.output.dense.weight\n",
      "bert.encoder.layer.11.output.dense.bias\n",
      "bert.encoder.layer.11.output.dense._input_quantizer._amax\n",
      "bert.encoder.layer.11.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.output.add_local_input_quantizer._amax\n",
      "bert.encoder.layer.11.output.add_residual_input_quantizer._amax\n",
      "bert.encoder.final_input_quantizer._amax\n",
      "bert.pooler.dense.weight\n",
      "bert.pooler.dense.bias\n",
      "qa_outputs.weight\n",
      "qa_outputs.bias\n"
     ]
    }
   ],
   "source": [
    "for key in state_dict.keys():\n",
    "    print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "58e75669",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(5.545955, dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_dict['bert.encoder.layer.0.attention.self.query._input_quantizer._amax']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9879771d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(6.7064476, dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_dict['bert.encoder.layer.0.attention.self.qv_a_input_quantizer._amax']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b037fb99",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "module, class, method, function, traceback, frame, or code object was expected, got builtin_function_or_method",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [27]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01minspect\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43minspect\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetsource\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquantized\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/inspect.py:1024\u001b[0m, in \u001b[0;36mgetsource\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m   1018\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetsource\u001b[39m(\u001b[38;5;28mobject\u001b[39m):\n\u001b[1;32m   1019\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the text of the source code for an object.\u001b[39;00m\n\u001b[1;32m   1020\u001b[0m \n\u001b[1;32m   1021\u001b[0m \u001b[38;5;124;03m    The argument may be a module, class, method, function, traceback, frame,\u001b[39;00m\n\u001b[1;32m   1022\u001b[0m \u001b[38;5;124;03m    or code object.  The source code is returned as a single string.  An\u001b[39;00m\n\u001b[1;32m   1023\u001b[0m \u001b[38;5;124;03m    OSError is raised if the source code cannot be retrieved.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1024\u001b[0m     lines, lnum \u001b[38;5;241m=\u001b[39m \u001b[43mgetsourcelines\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mobject\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1025\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(lines)\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/inspect.py:1006\u001b[0m, in \u001b[0;36mgetsourcelines\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m    998\u001b[0m \u001b[38;5;124;03m\"\"\"Return a list of source lines and starting line number for an object.\u001b[39;00m\n\u001b[1;32m    999\u001b[0m \n\u001b[1;32m   1000\u001b[0m \u001b[38;5;124;03mThe argument may be a module, class, method, function, traceback, frame,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1003\u001b[0m \u001b[38;5;124;03moriginal source file the first line of code was found.  An OSError is\u001b[39;00m\n\u001b[1;32m   1004\u001b[0m \u001b[38;5;124;03mraised if the source code cannot be retrieved.\"\"\"\u001b[39;00m\n\u001b[1;32m   1005\u001b[0m \u001b[38;5;28mobject\u001b[39m \u001b[38;5;241m=\u001b[39m unwrap(\u001b[38;5;28mobject\u001b[39m)\n\u001b[0;32m-> 1006\u001b[0m lines, lnum \u001b[38;5;241m=\u001b[39m \u001b[43mfindsource\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mobject\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1008\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m istraceback(\u001b[38;5;28mobject\u001b[39m):\n\u001b[1;32m   1009\u001b[0m     \u001b[38;5;28mobject\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mobject\u001b[39m\u001b[38;5;241m.\u001b[39mtb_frame\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/inspect.py:817\u001b[0m, in \u001b[0;36mfindsource\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m    809\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfindsource\u001b[39m(\u001b[38;5;28mobject\u001b[39m):\n\u001b[1;32m    810\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the entire source file and starting line number for an object.\u001b[39;00m\n\u001b[1;32m    811\u001b[0m \n\u001b[1;32m    812\u001b[0m \u001b[38;5;124;03m    The argument may be a module, class, method, function, traceback, frame,\u001b[39;00m\n\u001b[1;32m    813\u001b[0m \u001b[38;5;124;03m    or code object.  The source code is returned as a list of all the lines\u001b[39;00m\n\u001b[1;32m    814\u001b[0m \u001b[38;5;124;03m    in the file and the line number indexes a line in that list.  An OSError\u001b[39;00m\n\u001b[1;32m    815\u001b[0m \u001b[38;5;124;03m    is raised if the source code cannot be retrieved.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 817\u001b[0m     file \u001b[38;5;241m=\u001b[39m \u001b[43mgetsourcefile\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mobject\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    818\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m file:\n\u001b[1;32m    819\u001b[0m         \u001b[38;5;66;03m# Invalidate cache if needed.\u001b[39;00m\n\u001b[1;32m    820\u001b[0m         linecache\u001b[38;5;241m.\u001b[39mcheckcache(file)\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/inspect.py:697\u001b[0m, in \u001b[0;36mgetsourcefile\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m    693\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetsourcefile\u001b[39m(\u001b[38;5;28mobject\u001b[39m):\n\u001b[1;32m    694\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the filename that can be used to locate an object's source.\u001b[39;00m\n\u001b[1;32m    695\u001b[0m \u001b[38;5;124;03m    Return None if no way can be identified to get the source.\u001b[39;00m\n\u001b[1;32m    696\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 697\u001b[0m     filename \u001b[38;5;241m=\u001b[39m \u001b[43mgetfile\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mobject\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    698\u001b[0m     all_bytecode_suffixes \u001b[38;5;241m=\u001b[39m importlib\u001b[38;5;241m.\u001b[39mmachinery\u001b[38;5;241m.\u001b[39mDEBUG_BYTECODE_SUFFIXES[:]\n\u001b[1;32m    699\u001b[0m     all_bytecode_suffixes \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m importlib\u001b[38;5;241m.\u001b[39mmachinery\u001b[38;5;241m.\u001b[39mOPTIMIZED_BYTECODE_SUFFIXES[:]\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/site-packages/torch/package/package_importer.py:620\u001b[0m, in \u001b[0;36mpatched_getfile\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m    618\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mobject\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__module__\u001b[39m \u001b[38;5;129;01min\u001b[39;00m _package_imported_modules:\n\u001b[1;32m    619\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m _package_imported_modules[\u001b[38;5;28mobject\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__module__\u001b[39m]\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__file__\u001b[39m\n\u001b[0;32m--> 620\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_orig_getfile\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mobject\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/trans-fat/lib/python3.9/inspect.py:677\u001b[0m, in \u001b[0;36mgetfile\u001b[0;34m(object)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m iscode(\u001b[38;5;28mobject\u001b[39m):\n\u001b[1;32m    676\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mobject\u001b[39m\u001b[38;5;241m.\u001b[39mco_filename\n\u001b[0;32m--> 677\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmodule, class, method, function, traceback, frame, or \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    678\u001b[0m                 \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcode object was expected, got \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m    679\u001b[0m                 \u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mobject\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m))\n",
      "\u001b[0;31mTypeError\u001b[0m: module, class, method, function, traceback, frame, or code object was expected, got builtin_function_or_method"
     ]
    }
   ],
   "source": [
    "import inspect\n",
    "inspect.getsource(torch.ops.quantized.layer_norm)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trans-fat",
   "language": "python",
   "name": "trans-fat"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
